package com.huan.iteye.crawler;

import com.huan.iteye.domain.IteyeBlogTotalViews;
import com.huan.iteye.domain.IteyeBlogView;
import com.huan.iteye.properties.BlogProperties;
import com.huan.iteye.service.IteyeBlogTotalViewsService;
import com.huan.iteye.service.IteyeBlogViewService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.StringUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * iteye 博客爬取
 *
 * @author huan.fu
 * @date 2018/9/13 - 10:42
 */
@Component
@Slf4j
public class IteyeBlogCrawler {

	private static final Pattern REG_TOTAL_VIEWS = Pattern.compile("\\D* (\\d+)\\D*");

	@Autowired
	private BlogProperties blogProperties;
	@Autowired
	private IteyeBlogViewService blogViewService;
	@Autowired
	private IteyeBlogTotalViewsService blogTotalViewsService;

	/**
	 * 数据爬取
	 *
	 * @param workDate 爬取的日期
	 */
	public void crawler(String workDate) {
		if (StringUtils.isBlank(blogProperties.getCrawlerUrl())) {
			log.info("系统中没有配置要爬取博客的url,取消本地爬取");
			return;
		}
		long start = System.currentTimeMillis();
		try {
			Document document = Jsoup
					.connect(getCrawlerUrl(1))
					.header("Host", "huan1993.iteye.com")
					.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0")
					.get();
			saveTotalViews(document, workDate);
			TimeUnit.SECONDS.sleep(new Random(10).nextInt());
			int totalPage = totalPage(document);
			List<IteyeBlogView> blogViews = new ArrayList<>(100);
			for (int i = 1; i < totalPage + 1; i++) {
				String crawlerUrl = getCrawlerUrl(i);
				log.info("开始准备爬取:[{}]这一页的内容.", crawlerUrl);
				TimeUnit.SECONDS.sleep(new Random(50).nextInt());
				Connection connection = Jsoup
						.connect(crawlerUrl)
						.header("Host", "huan1993.iteye.com")
						.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0");
				if (i != 1) {
					connection.referrer("http://huan1993.iteye.com/");
				}
				document = connection.get();
				analysisOnePage(document, workDate, blogViews);
			}
			blogViewService.deleteBlogView(workDate);
			blogViewService.batchInsertBlogView(blogViews);
		} catch (Exception e) {
			log.error("爬取:[{}]日期的博客:[{}]出现异常.", workDate, blogProperties.getCrawlerUrl());
			log.error(e.getMessage(), e);
		} finally {
			long end = System.currentTimeMillis();
			log.info("完成本次爬取,耗时:[{}ms].", end - start);
		}
	}

	/**
	 * 保存每天总的博客访问总量
	 *
	 * @param document
	 * @param workDate
	 */
	private void saveTotalViews(Document document, String workDate) {
		blogTotalViewsService.deleteIteyeBlogTotalView(workDate);
		IteyeBlogTotalViews totalViews = new IteyeBlogTotalViews();
		totalViews.setWorkDate(workDate);
		Element element = document.selectFirst("#blog_actions ul li");
		Matcher matcher = REG_TOTAL_VIEWS.matcher(element.text());
		if (matcher.matches()) {
			totalViews.setViewCnt(Long.parseLong(matcher.group(1)));
		} else {
			totalViews.setViewCnt(0L);
		}
		blogTotalViewsService.addIteyeBlogTotalView(totalViews);

	}

	/**
	 * 获取访问数量
	 *
	 * @param text
	 * @return
	 */
	private long getViewNum(String text) {
		String reg = "\\d+";
		Pattern pattern = Pattern.compile(reg);
		Matcher matcher = pattern.matcher(text);
		if (matcher.find()) {
			return Long.parseLong(matcher.group());
		} else {
			return 0;
		}
	}

	/**
	 * 统计一页的博客
	 *
	 * @param document
	 */
	public void analysisOnePage(Document document, String workDate, List<IteyeBlogView> blogViews) {
		Elements elements = document.select("#main .blog_main");
		IteyeBlogView blogView;
		for (Element element : elements) {
			blogView = new IteyeBlogView();
			blogView.setTitle(element.select(".blog_title h3 a").text());
			blogView.setViewCnt(getViewNum(element.select(".blog_bottom ul li:eq(1)").text()));
			blogView.setWorkDate(workDate);
			blogViews.add(blogView);
		}
	}

	/**
	 * 获取爬取的url连接
	 *
	 * @param page
	 * @return
	 */
	public String getCrawlerUrl(int page) {
		if (page == 1) {
			return blogProperties.getCrawlerUrl().substring(0, blogProperties.getCrawlerUrl().indexOf('?'));
		} else {
			return String.format(blogProperties.getCrawlerUrl(), page);
		}
	}

	/**
	 * 获取总页数
	 *
	 * @param document
	 * @return
	 */
	private int totalPage(Document document) {
		Elements elements = document.getElementsByClass("pagination");
		if (!elements.isEmpty()) {
			Element element = elements.get(0);
			Elements a = element.select("a");
			return Integer.parseInt(a.get(a.size() - 2).text().trim());
		}
		return 0;
	}
}
